#! /usr/bin/env python
# -*- coding: utf-8 -*-


# Parlons Culture - Tools to generate and handle "Parlons Culture" OAI-PMH files
# By: Emmanuel Raviart <eraviart@easter-eggs.com>
#
# Copyright (C) 2010 Easter-eggs
# http://code.google.com/p/parlons-culture/
#
# This file is part of "Parlons Culture" project.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of the <organization> nor the
#       names of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


"""Generate static OAI-PMH XML file fromm CSV."""


import codecs
from ConfigParser import SafeConfigParser
import csv
import datetime
import itertools
import logging
from optparse import OptionParser
import os
import re
from string import Template
import sys
import time
import unicodedata
import urlparse


app_name = os.path.splitext(os.path.basename(__file__))[0]
column_number = None
conf = {}
language_re = re.compile(u'[a-z]{2}(\-[A-Z]{2})?')
line_number = None
log = logging.getLogger(app_name)
same_oai_identifiers_count = {}
xml_escapes = {
    '&' : '&amp;',
    '>' : '&gt;', 
    '<' : '&lt;', 
    '"' : '&#34;',
#    "'" : '&#39;',
    }


def add_to_entry(entry, label, value):
    error = False

    # Check and convert value.
    if value is None:
        return error
    value = value.strip() or None
    if value is None:
        return error
    if label == 'date':
        try:
            value = time.strptime(value, conf['csv_date_format'])
        except:
            if not error:
                log.error(u"""Ligne %d, colonne %d : La date "%s" n'est pas au format "%s".""" % (
                    line_number + 2, column_number + 1, value,  conf['csv_date_format']))
                error = True
        else:
            value = datetime.date(*value[:3])
    elif label == 'duree':
        try:
            value = time.strptime(value, conf['csv_duration_format'])
        except:
            if not error:
                log.error(u"""Ligne %d, colonne %d : La durée "%s" n'est pas au format "%s".""" % (
                    line_number + 2, column_number + 1, value,  conf['csv_duration_format']))
                error = True
        else:
            value = datetime.time(*value[3:5])
    elif label == 'langue':
        if language_re.match(value) is None and not error:
            log.error(u"""Ligne %d, colonne %d : La langue "%s" n'est pas au format "xx" ou "xx-YY".""" % (
                line_number + 2, column_number + 1, value))
            error = True
    elif label == 'types_de_ressources':
        value = {
            'audio': u'Audio',
            'video': u'Vidéo',
            'vignette': u'Vignette',
            }.get(simplify(value), value)

    # Store value in entry.
    if label in ('intervenants', 'mots_cles', 'partenaires', 'ressources', 'types_de_ressources'):
        # Multi-valued field
        entry.setdefault(label, []).append(value)
    else:
        assert label not in entry, """\
Le titre "%s" de la colonne apparaît pour la deuxième fois en colonne %d alors qu'il devrait être unique.\
""" % (label, column_number + 1)
        entry[label] = value

    return error


def load_environment(argv = None):
    """Configure the application environment."""
    if argv is None:
        argv = sys.argv
    parser = OptionParser("usage: %prog [options] config_file csv_file oai_pmh_file")
    parser.add_option('-c', '--encodage', dest = 'encoding',
        help = u'Encodage des caractères du fichier CSV (defaut : "utf-8")')
    parser.add_option('-D', '--duree', dest = 'csv_duration_format',
        help = u'Format de durée (defaut : "%H:%M:%S")')
    parser.add_option('-d', '--date', dest = 'csv_date_format',
        help = u'Format de date (defaut : "%Y/%m/%d")')
    parser.add_option('-s', '--separateur', dest = 'csv_separator',
        help = u'Délimiteur de colonnes CSV (defaut : ",")')
    parser.add_option('-v', '--verbeux', action = 'store_true', default = False, dest = 'verbose',
        help = u"Affiche plus de messages")
    options, arguments = parser.parse_args(argv[1:])
    if len(arguments) != 3:
        parser.error("Incorrect number of arguments")
    logging.basicConfig(level = logging.DEBUG if options.verbose else logging.WARNING, stream = sys.stdout)

    config_path = os.path.abspath(arguments[0])
    config_parser = SafeConfigParser(dict(here = os.path.dirname(config_path)))
    config_parser.read(config_path)

    config_encoding = (config_parser.get('ini', 'encodage') if config_parser.has_section('ini') else None) or 'utf-8'
    conf.update({
        'conference_publisher': config_parser.get('conference', 'editeur').decode(config_encoding),
        'csv_date_format': options.csv_date_format or config_parser.get('csv', 'format_date') or '%Y/%m/%d',
        'csv_duration_format': options.csv_duration_format or config_parser.get('csv', 'format_duree') or '%Y/%m/%d',
        'csv_encoding': options.encoding or config_parser.get('csv', 'encodage') or 'utf-8',
        'csv_file_path': os.path.normpath(arguments[1]),
        'csv_separator': options.csv_separator or config_parser.get('csv', 'separateur') or ',',
        'repo_admin_email': config_parser.get('depot', 'courriel_admin').decode(config_encoding),
        'repo_earliest_date': datetime.date(*time.strptime(config_parser.get('depot', 'date_debut'), '%Y-%m-%d')[:3]),
        'repo_name': config_parser.get('depot', 'nom').decode(config_encoding),
        'repo_url': config_parser.get('depot', 'url').decode(config_encoding),
        'xml_file_path': os.path.normpath(arguments[2]),
        })


def main(argv = None):
    load_environment(argv)

    csv_file = open(conf['csv_file_path'])
    csv_reader = csv.reader(csv_file, delimiter = conf['csv_separator'])

    labels = [
        {
            '': None,
            'collection': 'cycle_collection_exposition',
            'cycle': 'cycle_collection_exposition',
            'droit': 'droits',
            'exposition': 'cycle_collection_exposition',
            'intervenant': 'intervenants',
            'mot_cle': 'mots_cles',
            'mot_clef': 'mots_cles',
            'mots_clefs': 'mots_cles',
            'partenaire': 'partenaires',
            'ressource': 'ressources',
            'ressource_associee': 'ressources',
            'ressources_associees': 'ressources',
            'type_de_ressource': 'types_de_ressources',
            'type_de_ressource_associee': 'types_de_ressources',
            'type_ressource': 'types_de_ressources',
            'type_ressource_associee': 'types_de_ressources',
            'types_de_ressources_associees': 'types_de_ressources',
            'types_ressources': 'types_de_ressources',
            'types_ressources_associees': 'types_de_ressources',
            }.get(label, label)
        for label in (
            simplify(cell.decode(conf['csv_encoding'])) or None
            for cell in csv_reader.next()
            )
        ]

    entries = []
    global line_number
    for line_number, line in enumerate(csv_reader):
        row = [
            (cell.decode(conf['csv_encoding']).strip() or None) if cell else None
            for cell in line
            ]
        entry = {}
        error = False
        global column_number
        for column_number, (label, cell) in enumerate(itertools.izip(labels, row)):
            if label in ('intervenants', ):
                # Cell can contain multiple names (with commas).
                if cell is not None:
                    for value in cell.replace(u'/', u'|').replace(u'\r\n', u'|').replace(u'\r', u'|').replace(
                            u'\n', u'|').split(u'|'):
                        error = add_to_entry(entry, label, value) or error
            elif label in ('ressources'):
                # Cell can contain multiple URLs.
                if cell is not None:
                    for value in cell.replace(u'\r\n', u'|').replace(u'\r', u'|').replace(u'\n', u'|').split(u'|'):
                        error = add_to_entry(entry, label, value) or error
            elif label in ('mots_cles', 'partenaires', 'types_de_ressources'):
                # Cell can contain multiple values.
                if cell is not None:
                    for value in cell.replace(u'/', u'|').replace(u',', u'|').replace(u'\r\n', u'|').replace(
                            u'\r', u'|').replace(u'\n', u'|').split(u'|'):
                        error = add_to_entry(entry, label, value) or error
            else:
                error = add_to_entry(entry, label, cell) or error

        # Add default value to missing fields.
        entry.setdefault('langue', u'fr')
        if entry.get('editeur') is None and conf['conference_publisher'] is not None:
            entry['editeur'] = conf['conference_publisher']

        # Ensure that required values are present.
        if not error:
            for label in ('editeur', 'langue', 'page_web', 'theme', 'titre'):
                if entry.get(label) is None:
                    log.error(u"""Ligne %d : Valeur manquante pour le champ "%s".""" % (line_number + 2, label))
                    error = True
        if not error and len(entry.get('types_de_ressources', [])) != len(entry.get('ressources', [])):
            log.error(u"""\
Ligne %d : Le nombre de colonnes "Types de ressources" (%d) est différent du nombre de colonnes "Ressources" (%d).\
""" % (line_number + 2, len(entry.get('types_de_ressources', [])), len(entry.get('ressources', []))))
            error = True

        if not error:
            entries.append(entry)

    # Generate static OAI-PMH XML file.
    if entries:
        oai_pmh_file = codecs.open(conf['xml_file_path'], 'w', encoding = 'utf-8')
        for fragment in render_static_repository(entries):
            oai_pmh_file.write(fragment)
        oai_pmh_file.close()

    return 0


def render_record(entry):
    yield Template(u"""\
    <oai:record>
      <oai:header>
        <oai:identifier>$identifier</oai:identifier>
        <oai:datestamp>$date</oai:datestamp>
      </oai:header>
      <oai:metadata>
        <oai_dc:dc
            xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
            xmlns:dc="http://purl.org/dc/elements/1.1/"
            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
            xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/
                                http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
          <dc:title>$titre</dc:title>
""").substitute(
        date = xml_escape(entry['date'].isoformat()),
        identifier = xml_escape(url_to_oai_identifier(entry['page_web'])),
        titre = xml_escape(entry['titre']),
        )

    if entry.get('sous_titre') is not None:
        yield Template(u"""\
          <dc:title>$sous_titre</dc:title>
""").substitute(
            sous_titre = xml_escape(entry['sous_titre']),
            )
    elif entry.get('type') is not None:
        yield u"""\
          <dc:title/>
"""

    if entry.get('type') is not None:
        yield Template(u"""\
          <dc:title>$type</dc:title>
""").substitute(
            type = xml_escape(entry['type']),
            )

    yield Template(u"""\
          <dc:subject>$theme</dc:subject>
""").substitute(
        theme = xml_escape(entry['theme']),
        )

    if entry.get('sous_theme') is not None:
        yield Template(u"""\
          <dc:subject>$sous_theme</dc:subject>
""").substitute(
            sous_theme = xml_escape(entry['sous_theme']),
            )
    elif entry.get('mots_cles') is not None:
        yield u"""\
          <dc:subject/>
"""

    if entry.get('mots_cles') is not None:
        for mot_cle in entry['mots_cles']:
            yield Template(u"""\
          <dc:subject>$mot_cle</dc:subject>
""").substitute(
                mot_cle = xml_escape(mot_cle),
                )

    if entry.get('cycle_collection_exposition') is not None:
        yield Template(u"""\
          <dc:source>$cycle_collection_exposition</dc:source>
""").substitute(
            cycle_collection_exposition = xml_escape(entry['cycle_collection_exposition']),
            )
    elif entry.get('lieu') is not None:
        yield u"""\
          <dc:source/>
"""

    if entry.get('lieu') is not None:
        yield Template(u"""\
          <dc:source>$lieu</dc:source>
""").substitute(
            lieu = xml_escape(entry['lieu']),
            )

    if entry.get('intervenants') is not None:
        for intervenant in entry['intervenants']:
            yield Template(u"""\
          <dc:creator>$intervenant</dc:creator>
""").substitute(
                intervenant = xml_escape(intervenant),
                )

    if entry.get('resume') is not None:
        yield Template(u"""\
          <dc:description>$resume</dc:description>
""").substitute(
            resume = xml_escape(entry['resume']),
            )

    yield Template(u"""\
          <dc:date>$date</dc:date>
""").substitute(
        date = xml_escape(entry['date'].isoformat()),
        )

    if entry.get('duree') is not None:
        yield Template(u"""\
          <dc:format>$duree</dc:format>
""").substitute(
            duree = xml_escape(entry['duree'].strftime('%H:%M')),
            )

    if entry.get('langue') is not None:
        yield Template(u"""\
          <dc:language>$langue</dc:language>
""").substitute(
            langue = xml_escape(entry['langue']),
            )

    yield Template(u"""\
          <dc:identifier>$page_web</dc:identifier>
""").substitute(
        page_web = xml_escape(entry['page_web']),
        )

    if entry.get('ressources') is not None:
        for type, ressource in itertools.izip(entry['types_de_ressources'], entry['ressources']):
            yield Template(u"""\
          <dc:type>$type</dc:type>
          <dc:relation>$ressource</dc:relation>
""").substitute(
                ressource = xml_escape(ressource),
                type = xml_escape(type),
                )

    yield Template(u"""\
          <dc:publisher>$editeur</dc:publisher>
""").substitute(
        editeur = xml_escape(entry['editeur']),
        )

    if entry.get('partenaires') is not None:
        for partenaire in entry['partenaires']:
            yield Template(u"""\
          <dc:contributor>$partenaire</dc:contributor>
""").substitute(
                partenaire = xml_escape(partenaire),
                )

    if entry.get('droits') is not None:
        yield Template(u"""\
          <dc:rights>$droits</dc:rights>
""").substitute(
            droits = xml_escape(entry['droits']),
            )

    yield u"""\
        </oai_dc:dc>
      </oai:metadata>
    </oai:record>
"""


def render_static_repository(entries):
    yield Template(u"""\
<?xml version="1.0" encoding="utf-8"?>
<Repository xmlns="http://www.openarchives.org/OAI/2.0/static-repository"
            xmlns:oai="http://www.openarchives.org/OAI/2.0/"
            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
            xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/static-repository
                                http://www.openarchives.org/OAI/2.0/static-repository.xsd">
  <Identify>
    <oai:repositoryName>$name</oai:repositoryName>
    <oai:baseURL>$url</oai:baseURL>
    <oai:protocolVersion>2.0</oai:protocolVersion>
    <oai:adminEmail>$admin_email</oai:adminEmail>
    <oai:earliestDatestamp>$earliest_date</oai:earliestDatestamp>
    <oai:deletedRecord>no</oai:deletedRecord>
    <oai:granularity>YYYY-MM-DD</oai:granularity>
  </Identify>
  <ListMetadataFormats>
    <oai:metadataFormat>
      <oai:metadataPrefix>oai_dc</oai:metadataPrefix>
      <oai:schema>http://www.openarchives.org/OAI/2.0/oai_dc.xsd</oai:schema>
      <oai:metadataNamespace>http://www.openarchives.org/OAI/2.0/oai_dc/</oai:metadataNamespace>
    </oai:metadataFormat>
  </ListMetadataFormats>
  <ListRecords metadataPrefix="oai_dc">
""").substitute(
        admin_email = xml_escape(conf['repo_admin_email']),
        earliest_date = xml_escape(conf['repo_earliest_date'].isoformat()),
        name = xml_escape(conf['repo_name']),
        url = xml_escape(conf['repo_url']),
        )
    for entry in entries:
        for fragment in render_record(entry):
            yield fragment
    yield u"""\
  </ListRecords>
</Repository>"""


def simplify(s, separator = '_'):
    """Convert unicode string to a slug."""
    return ''.join(
        key if key == separator else ''.join(group)
        for key, group in itertools.groupby(
            c if c in '0123456789abcdefghijklmnopqrstuvwxyz' else separator
            for c in unicodedata.normalize('NFD', s).encode('ASCII', 'ignore').lower()
            )
        ).strip(separator)


def url_to_oai_identifier(url):
    """Convert an URL to a OAI Identifier and ensure that it is unique.

    cf http://www.openarchives.org/OAI/2.0/guidelines-oai-identifier.htm for OAI identifier format.
    """
    split_url = urlparse.urlsplit(url)
    oai_identifier = u'oai:%s:%s' % (split_url[1], u'-'.join(
        slug
        for slug in (
            simplify(unicode(fragment), separator = '-')
            for fragment in split_url[2:]
            )
        if slug
        ))
    # Since several conferences can share the same URL, ensure that their OAI idenfier are distinct by adding and
    # index to them.
    count = same_oai_identifiers_count.get(oai_identifier, 0)
    same_oai_identifiers_count[oai_identifier] = count + 1
    if count == 0:
        return oai_identifier
    else:
        return u'%s-%d' % (oai_identifier, count)


def xml_escape(s):
    return re.sub(r"""([&<">])""", lambda m: xml_escapes[m.group()], s)


if __name__ == "__main__":
    sys.exit(main())
